In [1]:
import numpy as np
import pandas as pd
from sklearn import metrics
import matplotlib.pyplot as plt
%matplotlib inline
In [183]:
df = pd.read_excel("C:\\Users\\vallabh\\Vallabha Datta\\Project\\df_reg.xlsx", delimiter=',')
df.head(5)
Out[183]:
In [184]:
df = df.drop('Unnamed: 0', axis=1)
df.head(5)
Out[184]:
In [185]:
# Checking for data types and null values
df.info()
From above info, there are 8 null values in "State" column
In [186]:
state_mean = int(df.State.mean())
state_mean
Out[186]:
Replacing null value with mean For Montana cat code is 27, So, replacing missing state with Montana df.State.fillna(state_mean, inplace=True)
In [187]:
df.State.fillna(state_mean, inplace=True)
In [188]:
df.info()
In [189]:
df.shape
Out[189]:
In [190]:
df['Unemployment_rate_2018'].describe()
Out[190]:
In [191]:
# Changing Unemployment_rate_2018 to category to perform knn
# Dividing into 5 Categories
lst = []
for i in df.Unemployment_rate_2018:
if i<=3.1:
lst.append("Very low") # If Unemployment_rate is less than 3.1, then it is very low
elif i>3.1 and i<=3.9:
lst.append("Low") # If Unemployment_rate is between 3.1 & 3.9[including 3.9] then it is low
elif i>3.9 and i<=4.9:
lst.append("Average") # If Unemployment_rate is between 3.9 & 4.9[including 4.9] then it is Average
elif i>4.9 and i<=10:
lst.append("High") # If Unemployment_rate is between 4.9 & 10[including 10] then it is High
else:
lst.append("Very High") # If Unemployment_rate is greater than 10, then it is very high
lst
Out[191]:
In [192]:
df.Unemployment_rate_2018 = lst
In [193]:
df['Unemployment_rate_2018'].head(5)
Out[193]:
In [194]:
# Selecting Unemployment_rate_2018 as target variable
df_target = df['Unemployment_rate_2018']
df = df.drop('Unemployment_rate_2018', axis =1)
In [195]:
from sklearn import preprocessing
In [196]:
# Performing min-max scaling
min_max_scaler = preprocessing.MinMaxScaler()
min_max_scaler.fit(df)
Out[196]:
In [197]:
df_scaled = min_max_scaler.fit_transform(df)
df_scaled
Out[197]:
In [198]:
df_scaled = pd.DataFrame(df_scaled, columns=df.columns)
In [199]:
df_scaled.head(5)
Out[199]:
In [200]:
np.set_printoptions(precision=2, linewidth=100)
In [201]:
# Splitting data into training and testing
from sklearn.model_selection import train_test_split
df_train, df_test, df_target_train, df_target_test = train_test_split(df_scaled, df_target, test_size=0.2, random_state=478)
In [202]:
df_target_train.head(5)
Out[202]:
In [203]:
df_target_test.head(5)
Out[203]:
In [204]:
df_train.head(5)
Out[204]:
In [205]:
df_test.head(5)
Out[205]:
In [206]:
print("Shape of Train data: ", df_train.shape)
print("Shape of Test data: ", df_test.shape)
In [207]:
df_train_arr = np.array(df_train)
df_test_arr = np.array(df_test)
df_target_train_arr = np.array(df_target_train)
df_target_test_arr = np.array(df_target_test)
In [208]:
def knn_search(x, D, K, measure):
""" find K nearest neighbors of an instance x among the instances in D """
if measure == 0:
# euclidean distances from the other points
dists = np.sqrt(((D - x)**2).sum(axis=1))
elif measure == 1:
# first find the vector norm for each instance in D as wel as the norm for vector x
D_norm = np.array([np.linalg.norm(D[i]) for i in range(len(D))])
x_norm = np.linalg.norm(x)
# Compute Cosine: divide the dot product o x and each instance in D by the product of the two norms
sims = np.dot(D,x)/(D_norm * x_norm)
# The distance measure will be the inverse of Cosine similarity
dists = 1 - sims
idx = np.argsort(dists) # sorting
# return the indexes of K nearest neighbors
return idx[:K], dists
In [209]:
neigh_idx, distances = knn_search(df_test_arr[0], df_train_arr, 5, 0)
In [210]:
print(neigh_idx)
print("\nNearest Neigbors:")
df_train.iloc[neigh_idx]
Out[210]:
In [211]:
# printing distances of top 5 nearest neighbors
print(distances[neigh_idx])
In [212]:
neigh_labels = df_target_train_arr[neigh_idx]
print(neigh_labels)
Top 5 nearest neighbors are: High, Average, High, Average, Average
In [213]:
from collections import Counter
print(Counter(neigh_labels))
In [214]:
Counter(neigh_labels).most_common(1)
Out[214]:
Since, majority of votes are Average for top 5 nearest neighbours, the predicted value of unemployment rate will fall under Average category
In [215]:
X_train, X_test, y_train, y_test = train_test_split(df_scaled, df_target, test_size=0.2, random_state=50)
In [216]:
from sklearn.neighbors import KNeighborsClassifier
Using KNN classifier
In [217]:
knn = KNeighborsClassifier(n_neighbors=5)
In [218]:
knn.fit(X_train, y_train)
Out[218]:
In [219]:
# Performing prediction
pred = knn.predict(X_test)
In [220]:
from sklearn.metrics import classification_report, confusion_matrix
In [105]:
print(confusion_matrix(y_test, pred))
print(classification_report(y_test, pred))
In [221]:
error_rate = []
for i in range(1,40):
knn= KNeighborsClassifier(n_neighbors = i)
knn.fit(X_train, y_train)
pred_i = knn.predict(X_test)
error_rate.append(np.mean(pred_i != y_test))
In [222]:
import matplotlib.pyplot as plt
plt.figure(figsize=(10,6))
plt.plot(range(1,40), error_rate, color='blue', linestyle='dashed', marker = 'o', markerfacecolor='red', markersize=10)
plt.title('Error Rate vs K value')
plt.xlabel('K')
plt.ylabel('Error Rate')
Out[222]:
From the Error vs K plot, the best value of k is 3. So, performing prediction with 3 neighbors
In [738]:
knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(X_train, y_train)
pred_knn = knn.predict(X_test)
print(confusion_matrix(y_test, pred))
print(classification_report(y_test, pred))
Accuracy using knn classifier is 72%
2) USING TERM DOCUMENT CATEGORIZATION
In [223]:
df_train.shape
Out[223]:
In [224]:
numTerms=df_train.shape[0]
NDocs = df_train.shape[1]
In [225]:
termFreqs = df_train.sum(axis=1)
termFreqs.head(5)
Out[225]:
In [226]:
plt.plot(sorted(termFreqs, reverse=True))
plt.show()
In [227]:
DF = pd.DataFrame([(df_train!=0).sum(1)]).T
DF.head(5)
Out[227]:
In [228]:
NMatrix=np.ones(np.shape(df_train), dtype=float)*NDocs
np.set_printoptions(precision=2,suppress=True,linewidth=120)
print(NMatrix)
In [229]:
# Convert each entry into IDF values
# IDF is the log of the inverse of document frequency
# Note that IDF is only a function of the term, so all columns will be identical.
IDF = np.log2(np.divide(NMatrix, np.array(DF)))
In [230]:
IDF
Out[230]:
In [231]:
TD_tfidf = df_train * IDF
In [232]:
TD_tfidf.head(10)
Out[232]:
In [233]:
IDF.T[0].shape
Out[233]:
Converting test data using TD*iDF
In [234]:
numTerms2=df_test.shape[0]
NDocs2 = df_test.shape[1]
In [235]:
DF2 = pd.DataFrame([(df_test!=0).sum(1)]).T
DF2.head(10)
Out[235]:
In [236]:
NMatrix2=np.ones(np.shape(df_test), dtype=float)*NDocs2
np.set_printoptions(precision=2,suppress=True,linewidth=120)
print(NMatrix2)
In [237]:
IDF2 = np.log2(np.divide(NMatrix2, np.array(DF2)))
IDF2
Out[237]:
In [238]:
df2_tfidf = df_test * IDF2
In [239]:
df2_tfidf.head(5)
Out[239]:
In [240]:
error_rate = []
for i in range(1,40):
knn= KNeighborsClassifier(n_neighbors = i)
knn.fit(df_train, df_target_train)
pred_i = knn.predict(df_test)
error_rate.append(np.mean(pred_i != df_target_test))
In [241]:
plt.figure(figsize=(10,6))
plt.plot(range(1,40), error_rate, color='blue', linestyle='dashed', marker = 'o', markerfacecolor='red', markersize=10)
plt.title('Error Rate vs n_neighbors')
plt.xlabel('n_neighbors')
plt.ylabel('Error Rate')
Out[241]:
In [242]:
# From the above graph the best k value is 10
knn = KNeighborsClassifier(n_neighbors=10)
In [243]:
knn.fit(TD_tfidf, df_target_train)
Out[243]:
In [244]:
pred = knn.predict(df2_tfidf)
In [546]:
print(confusion_matrix(df_target_test, pred))
print(classification_report(df_target_test, pred))
In [732]:
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(df_target_test, pred))
From the above classification report, Wei_Ave using TD*IDF = 0.61, Wei_Ave using Knn = 0.73. So, knn model without using TD_IDF is best in this case.
Predicting the Unemploment_rate for random query
In [245]:
import random
x=[]
for i in range(1,29):
y = random.random()
x.append(y)
In [246]:
# Each term in query x must be multiplied by the idf value of the term we computed earlier (the IDF matrix)
x_tfidf = x * IDF[0] # note that this coordinatewise multiplication of two vectors
print(x_tfidf)
In [247]:
x_tfidf.shape
Out[247]:
In [248]:
DT_tfidf = TD_tfidf
DT_array = np.array(DT_tfidf)
In [249]:
DT_array.shape
Out[249]:
In [250]:
df.head(5)
Out[250]:
In [251]:
DT_array
Out[251]:
In [253]:
neigh_idx, distances = knn_search(x_tfidf, DT_array, 5, 0)
In [261]:
# Distances between query objects and training objects
distances
Out[261]:
In [262]:
distances = pd.Series(distances, index=DT_tfidf.index)
In [263]:
print("Query:", x)
print("\nNeighbors:")
DT_tfidf.iloc[neigh_idx]
Out[263]:
In [264]:
df_target.shape
Out[264]:
In [265]:
cat_labels = np.array(df_target_train)
cat_labels = pd.Series(cat_labels, index=DT_tfidf.index)
DT_tfidf["Category"] = cat_labels
In [266]:
def knn_classify(x, D, K, labels, measure):
from collections import Counter
neigh_idx, distances = knn_search(x, D, K, measure)
neigh_labels = labels[neigh_idx]
count = Counter(neigh_labels)
print("Labels for top ", K, "neighbors: ", count)
return count.most_common(1)[0][0]
In [267]:
print("Instance to classify:\n", x)
print("Predicted Category for the new instance: ", knn_classify(x_tfidf, DT_array, 5, cat_labels, 0))
Predicted category for the new instance using TD*IDF is "High"
3) USING DECISION TREE CLASSIFIER
In [268]:
from sklearn.tree import DecisionTreeClassifier
dtree = DecisionTreeClassifier()
In [269]:
dtree.fit(df_train,df_target_train)
Out[269]:
In [270]:
predictions = dtree.predict(df_test)
In [271]:
print(dtree.score(df_test, df_target_test))
In [272]:
print(dtree.score(df_train, df_target_train))
Tree performed well for training set than testing set. Tree score for training set is 1 which is not possible practically. So, we can infer that the decision tree model is too simple and underfitted
In [552]:
print(classification_report(df_target_test,predictions))
Accuracy using decision tree is 72%
Setting criterion to entropy
In [273]:
dtree = DecisionTreeClassifier(criterion = "entropy")
In [274]:
dtree.fit(df_train,df_target_train)
Out[274]:
In [275]:
predictions = dtree.predict(df_test)
print(classification_report(df_target_test,predictions))
After setting criterion as "entropy", I got accuracy as 76%. Which is better than "gini" criterion
Changing default values in decision tree classifier
In [276]:
error_rate = []
for i in range(1,40):
dtree = DecisionTreeClassifier(criterion = "entropy", max_depth = i)
dtree.fit(df_train,df_target_train)
predictions = dtree.predict(df_test)
error_rate.append(np.mean(predictions != df_target_test))
In [277]:
plt.figure(figsize=(10,6))
plt.plot(range(1,40), error_rate, color='blue', linestyle='dashed', marker = 'o', markerfacecolor='red', markersize=10)
plt.title('Error Rate vs max_depth')
plt.xlabel('max_depth')
plt.ylabel('Error Rate')
Out[277]:
In [278]:
# from the above plot, best value of max_depth =13
dtree = DecisionTreeClassifier(criterion = "entropy",max_depth = 13)
In [279]:
t = dtree.fit(df_train,df_target_train)
In [673]:
predictions = dtree.predict(df_test)
print(classification_report(df_target_test,predictions))
In [731]:
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(df_target_test, predictions))
So, by using decision tree classifier the best accuracy achieved is 78%
In [280]:
from IPython.display import Image
from sklearn.externals.six import StringIO
from sklearn.tree import export_graphviz
import pydot
features = list(df.columns[0:])
features
Out[280]:
Printing and exporting decision tree
In [691]:
dot_data = StringIO()
export_graphviz(dtree, out_file=dot_data,feature_names=features,filled=True,rounded=True)
graph = pydot.graph_from_dot_data(dot_data.getvalue())
Image(graph[0].create_png())
Out[691]:
4)USING NAIVE BAYES (GAUSSIAN) CLASSIFIER
In [281]:
from sklearn import naive_bayes
nb_model = naive_bayes.GaussianNB()
nb_model = nb_model.fit(df_train, df_target_train)
nb_pred = nb_model.predict(df_test)
nb_pred[1:20]
Out[281]:
In [282]:
print(nb_model.score(df_test, df_target_test))
In [283]:
print(nb_model.score(df_train, df_target_train))
In [284]:
confusion_mat = confusion_matrix(df_target_test, nb_pred)
print(confusion_mat)
In [474]:
print(classification_report(df_target_test, nb_pred))
In [730]:
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(df_target_test, nb_pred))
Accuracy using Naive Bayes is 54.4%
5) USING RANDOM FOREST CLASSIFIER
In [285]:
from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=100,criterion='entropy', random_state=None)
rfc.fit(df_train, df_target_train)
Out[285]:
In [286]:
rfc_pred = rfc.predict(df_test)
In [287]:
print(confusion_matrix(df_target_test,rfc_pred))
In [288]:
# Model Accuracy, how often is the classifier correct?
print("Accuracy:",metrics.accuracy_score(df_target_test, rfc_pred))
By using default values for RandomForestClassifier we got 78.69% accuracy
In [289]:
error_rate = []
for i in range(1,40):
rfc = RandomForestClassifier(n_estimators=100,criterion='entropy', random_state=None, max_depth=i)
rfc.fit(df_train, df_target_train)
rfc_pred = rfc.predict(df_test)
error_rate.append(np.mean(rfc_pred != df_target_test))
In [290]:
plt.figure(figsize=(10,6))
plt.plot(range(1,40), error_rate, color='blue', linestyle='dashed', marker = 'o', markerfacecolor='red', markersize=10)
plt.title('Error Rate vs max_depth')
plt.xlabel('max_depth')
plt.ylabel('Error Rate')
Out[290]:
In [349]:
# best value of max_depth is 40
rfc = RandomForestClassifier(n_estimators=100,criterion='entropy', random_state=None,max_depth=40)
rfc.fit(df_train, df_target_train)
Out[349]:
In [350]:
rfc_pred = rfc.predict(df_test)
In [351]:
print(confusion_matrix(df_target_test,rfc_pred))
print("Accuracy:",metrics.accuracy_score(df_target_test, rfc_pred))
By setting n_estimators to 100,criterion to 'entropy', random_state to None and max_depth to 40, the maximum accuracy achieved is 80.7%
Of all of the above classification techniques, Random Forest Classifier has maximum accuracy. So, let's perform prediction using random forest
In [ ]: